#!/bin/bash
day=$1
if [ -z $day ]; then
    day=`date -d last-day +%Y%m%d`
fi

scp -r lichunlong@124.250.26.88:/home/lichunlong/CarMaster/dmarket_access_log/$day dmarket_access_log/.

dir="dmarket_access_log/${day}/"
gzfilelist=`ls ${dir}`
for file in $gzfilelist
do
    #echo $file
    gzip -dv ${dir}${file} ${dir}${day}-dmarket_access_log
done

filelist=`ls ${dir}`
mergeDir=merge_dmarket_access_log
touch "${mergeDir}\${day}-dmarket_access_log"
for file in $filelist
do
    `cat ${dir}${file} >> ${mergeDir}/${day}-dmarket_access_log`
done

statDir=stat_user_visit_count
cat ${mergeDir}/${day}-dmarket_access_log | awk -F\" '{print $6}' |  awk -F\/ '{print $11}' | awk -F\( '{print $1}' | sort |uniq -c | sort -n > $statDir/$day-ua-stat

python format_url.py $day
echo "format url finished"
python pvLoadToDb.py $day
echo "load finished"
python urlLoad.py $day
echo "every url load finished"
